/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.crawwle.career.crawler;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import net.crawwle.career.crawler.controller.CareerController;

import org.apache.log4j.Logger;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

/**
 * @author Yasser Ganjisaffar <lastname at gmail dot com>
 */
public class CareerCrawlController {


	protected static final Logger logger = Logger.getLogger(CareerCrawlController.class.getName());

	public static void main(String[] args) throws Exception {
		if (args.length != 3) {
			System.out.println("Needed parameters: ");
			System.out.println("\t rootFolder (it will contain intermediate crawl data)");
			System.out.println("\t numberOfCralwers (number of concurrent threads)");
			System.out.println("\t numberOfCralwers (number of Depth)");
			logger.info("Needed parameters:");
			logger.info("\t rootFolder (it will contain intermediate crawl data)");
			logger.info("\t numberOfCralwers (number of concurrent threads)");
			logger.info("\t numberOfCralwers (number of Depth)");
			return;
		}

		logger.info("-----parameters-----");
		logger.info("\t rootFolder (it will contain intermediate crawl data)=" + args[0]);
		logger.info("\t numberOfCralwers (number of concurrent threads)=" + args[1]);
		logger.info("\t numberOfCralwers (number of Depth)=" + args[2]);

		/*
		 * crawlStorageFolder is a folder where intermediate crawl data is
		 * stored.
		 */
		String crawlStorageFolder = args[0];

		/*
		 * numberOfCrawlers shows the number of concurrent threads that should
		 * be initiated for crawling.
		 */
		int numberOfCrawlers = Integer.parseInt(args[1]);

		CrawlConfig config = new CrawlConfig();

		config.setCrawlStorageFolder(crawlStorageFolder);

		logger.info("\t Proxyサーバ設置 (/opt/conf/crawwle.properties)");
		//Proxyサーバ
		Properties conf = new Properties();
        conf.load(new FileInputStream("/opt/conf/crawwle.properties"));
        String proxyHost = conf.getProperty("proxy.host");
        if(proxyHost != null && !"".equals(proxyHost)){
            config.setProxyHost(proxyHost);
            int port = Integer.parseInt(conf.getProperty("proxy.port"));
            config.setProxyPort(port);
    		System.setProperty("proxySet", "true");
    		System.setProperty("proxyHost", proxyHost);
    		System.setProperty("proxyPort", conf.getProperty("proxy.port"));
    		logger.info("\t Proxyサーバ=" + proxyHost + ":" + conf.getProperty("proxy.port"));
        }
		//config.setProxyHost("10.221.237.10");
		//config.setProxyPort(8080);



		logger.info("add seed urls from /opt/conf/*.txt ");

		String path = "/opt/conf";
		File dir = new File(path);
		File[] files = dir.listFiles();
		List<String> urlList = new ArrayList<String>();
		Map<String,String> urlMap = new HashMap<String,String>();
		for (int i = 0; i < files.length; i++) {
			File file = files[i];
			String fileName = file.getName();
			logger.info("seed file:" + file);
			int point = fileName.lastIndexOf(".txt");
			if (point == -1) {
				continue;
			}
			BufferedReader br = new BufferedReader(new FileReader(file));
			String str = br.readLine();
			while (str != null) {

				if (str.toLowerCase().indexOf("http://") >= 0
						|| str.toLowerCase().indexOf("https://") >= 0) {
					String[] strList = str.split("\t");
					urlList.add(strList[0]);
					if(strList.length > 1){
						urlMap.put(strList[1], strList[0]);
					}
				}
				str = br.readLine();
			}

			br.close();
		}

		if(urlList.size() == 0){
			logger.warn("There does not exist urls in /opt/conf/*.txt");
			return;
		}



		/*
		 * Be polite: Make sure that we don't send more than 1 request per
		 * second (1000 milliseconds between requests).
		 */
		config.setPolitenessDelay(1000);

		/*
		 * You can set the maximum crawl depth here. The default value is -1 for
		 * unlimited depth
		 */
		config.setMaxDepthOfCrawling(Integer.parseInt(args[2]));

		/*
		 * You can set the maximum number of pages to crawl. The default value
		 * is -1 for unlimited number of pages
		 */
		config.setMaxPagesToFetch(-1);

		/*
		 * Do you need to set a proxy? If so, you can use:
		 * config.setProxyHost("proxyserver.example.com");
		 * config.setProxyPort(8080);
		 *
		 * If your proxy also needs authentication:
		 * config.setProxyUsername(username); config.getProxyPassword(password);
		 */

		/*
		 * This config parameter can be used to set your crawl to be resumable
		 * (meaning that you can resume the crawl from a previously
		 * interrupted/crashed crawl). Note: if you enable resuming feature and
		 * want to start a fresh crawl, you need to delete the contents of
		 * rootFolder manually.
		 */
		config.setResumableCrawling(false);

		/*
		 * Instantiate the controller for this crawl.
		 */
		PageFetcher pageFetcher = new PageFetcher(config);
		RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
		RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
		CareerController controller = new CareerController(config, pageFetcher, robotstxtServer);
		controller.setUrlMap(urlMap);

		/*
		 * For each crawl, you need to add some seed urls. These are the first
		 * URLs that are fetched and then the crawler starts following links
		 * which are found in these pages
		 */

		logger.info("\t seed urls=");
		for (int i = 0; i < urlList.size(); i++) {

			logger.info("\t\t " + urlList.get(i));
			controller.addSeed(urlList.get(i).toString());
		}
		//controller.addSeed("http://www.hatalike.jp/PLA_001/JT_01/?SCA=01&st=P00&FF=1");
		//controller.addSeed("http://employment.en-japan.com/k_yokohama/t_gaishi/");
		//controller.addSeed("http://next.rikunabi.com/new/");
		//controller.addSeed("http://employment.en-japan.com/list_special/");
		//controller.addSeed("http://next.rikunabi.com/area_wp0300000000/new/");
		//controller.addSeed("http://www.workport.co.jp/new/");


		logger.info("------------Start the crawl!------------");

		/*
		 * Start the crawl. This is a blocking operation, meaning that your code
		 * will reach the line after this only when crawling is finished.
		 */
		controller.start(CareerCrawler.class, numberOfCrawlers);
	}
}
